package cn.edu.ruc.dbiir.htmlparser;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.BasicConfigurator;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.alibaba.fastjson.JSONObject;
/**
 * html解析
* author：mark   
* createTime：Nov 26, 2019 11:03:16 AM   
* @version
 */
public class ParseHtmlJob {
	public static class HtmlMapper extends Mapper<LongWritable,Text, LongWritable, Text>{
		protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			
			Document document = Jsoup.parse(value.toString());
			JSONObject entity = new JSONObject();
			entity.put("title", document.getElementsByTag("title").text());
			entity.put("explain", Jsoup.parse(document.getElementsByClass("lemma-summary").html()).getElementsByClass("para").text());
//			System.out.println(document.getElementsByTag("title").text());
//			System.out.println(Jsoup.parse(document.getElementsByClass("lemma-summary").html()).getElementsByClass("para").text());
			context.write(key, new Text(entity.toJSONString()));
		}
	}

	public static void main(String[] args) {
		try {

			//            Configuration conf=new Configuration();
			//            Job job=Job.getInstance(conf,"Test lineRecordReader");
			//            job.setJarByClass(TestUserInputFormat.class);
			BasicConfigurator.configure();
			Configuration conf = new Configuration();
			conf.set("fs.defaultFS", "hdfs://db-01:9000");
			conf.set("mapreduce.framework.name", "yarn");
			conf.set("yarn.resourcemanager.hostname", "db-01");
			//1.创建一个job和任务入口
			Job job = Job.getInstance(conf);

			//    		Job job = Job.getInstance(new Configuration());

			job.setJarByClass(ParseHtmlJob.class);
			((JobConf)job.getConfiguration()).setJar("/Users/mark/git/htmlmr/target/mrtest-0.0.1-SNAPSHOT-jar-with-dependencies.jar");

			job.setInputFormatClass(HtmlInputFormat.class);
			job.setMapperClass(HtmlMapper.class);
			job.setOutputKeyClass(LongWritable.class);
			job.setOutputValueClass(Text.class);
			FileInputFormat.addInputPath(job, new Path("hdfs://db-01:9000/test/bkhtml"));
			FileOutputFormat.setOutputPath(job, new Path("hdfs://db-01:9000/test/mre7"));
			System.exit(job.waitForCompletion(true) ? 0 : 1);
		} catch (IllegalStateException e) {
			e.printStackTrace();
		} catch (IllegalArgumentException e) {
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
}
